import json
import logging
import os
import stat
import time
import uuid
from ftplib import FTP
import queue
import threading
from service import DBSql

class FTPClient():
    def __init__(self, task_id, ip_id):
        self.db = DBSql()
        self.task_id = task_id
        self.ip_id = ip_id
        self.dir_queue = queue.Queue()
        self.download_file_queue = queue.Queue()
        self.sql_insert_queue = queue.Queue()
        self.service = 'ftp'
        self.encoding = 'utf-8'
        sql = "SELECT * FROM `ip_list_%s` WHERE `id` = %s"
        with self.db.query(sql_text=sql, params=(self.task_id, self.ip_id)) as info:
            pass

        if info:
            config = info.get(f'{self.service}_config')
            config = json.loads(config)
            self.hostname = config.get('hostname')
            self.port = int(config.get('port'))
            self.username = config.get('username')
            self.password = config.get('password')
            self.paths = config.get('path')
            for path in self.paths:
                self.dir_queue.put(path)
            
        self.task_uuid = str(uuid.uuid4())
        self.task_folder_path = './save_files/' + self.task_uuid
        if not os.path.exists(self.task_folder_path):
            os.makedirs(self.task_folder_path)
        self.save_file_type = ['pdf', 'doc', 'docx', 'txt']
        self.scaned_path = set()
        self.downloaded_file = set()
        self.scaned_path_nums = 0
        self.thread_nums = 10
        self.thread_running_flag = {} # 用于标记线程是否在运行
        self.skip_path = ['/proc', '/sys', '公司/artical_scan', '.local/share/Trash', '.venv', '/smb-share']
        self.sql_insert_num = 20
        self.can_connect = True
    
    def create_client(self):
        try:
            ftp = FTP()
            ftp.connect(self.hostname, self.port, timeout=10)
            ftp.login(self.username, self.password)
            ftp.encoding = self.encoding 
            logging.info(ftp.getwelcome())
            return ftp
        except Exception as e:
            logging.error(e)
            logging.error(f'进程运行情况：{self.thread_running_flag.values()}')
            self.can_connect = False
            return None

    
    def list_dir_info(self, ftp, path):
        dir_list = []
        res = []
        def add_in_list(line):
            return dir_list.append(line)
        ftp.dir(path, add_in_list)
        if dir_list:
            for dir_info in dir_list:
                data = dir_info.split()
                res.append({
                    'filename': data[8],
                    'type': 'dir' if data[0].startswith('d') else 'link' 
                                if data[0].startswith('l') else 'file' 
                                if data[0].startswith('-') else 'unknown',
                    'size': data[4],
                })
        return res
    
    def save_file_info(self, file_name, file_uuidname, file_save_path, file_path, file_format, file_size):
        sql = """
            INSERT INTO `file_info_%s` 
            (`file_name`, `file_uuidname`, `file_save_path`, `file_path`, `file_format`, `file_size`, `service`, `ip_id`, `port`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
        """
        with self.db.insert(sql_text=sql, params=(self.task_id, file_name, 
                                                  file_uuidname, file_save_path, 
                                                  file_path, file_format, file_size,
                                                    self.service, self.ip_id, self.port)):
            pass
    
    def save_file_info_many(self, data:list):
        # file_name, file_uuidname, file_save_path, file_path, file_format, file_size = data[0]
        data_len = len(data)
        sql = """
            INSERT INTO `file_info_%s` 
            (`file_name`, `file_uuidname`, `file_save_path`, `file_path`, `file_format`, `file_size`, `service`, `ip_id`, `port`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
        """+",(%s, %s, %s, %s, %s, %s, %s, %s, %s)"*(data_len-1)
        params = [self.task_id]
        for item in data:
            item.extend([self.service, self.ip_id, self.port])
            params.extend(item)
        with self.db.insert(sql_text=sql, params=tuple(params)) as insert_id:
            logging.info(f'批量插入文件信息成功，插入数量：【{data_len}】，id：{insert_id}')
            pass
    
    def get_files(self, remote_path, ftp):
        logging.info(f'扫描文件夹：【{remote_path}】中...')
        for path_ in self.skip_path:
            if path_ in remote_path:
                return
        self.scaned_path.add(remote_path)
        for file in self.list_dir_info(ftp, remote_path):
            file_name = file.get('filename')
            file_path = os.path.join(remote_path, file_name)
            flag = file.get('type')
            file_size = file.get('size')
            if flag == 'dir' and file_path not in self.scaned_path:
                self.dir_queue.put(file_path)
            elif flag == 'file':
                file_format = file_name.split('.')[-1]
                if file_format in self.save_file_type and file_path not in self.downloaded_file:
                    self.download_file_queue.put((file_path, file_size))
                    # self.download_file(file_path, file_size)
        self.scaned_path_nums += 1
    
    def download_file(self, remote_path, file_size, ftp):
        logging.info(f'下载文件：【{remote_path}】中...')
        file_name = remote_path.split('/')[-1]
        file_format = file_name.split('.')[-1]
        file_uuidname = str(uuid.uuid4()) + '.' + file_format
        local_path = os.path.join(self.task_folder_path,file_uuidname)
        try:
            # 下载文件
            with open(local_path, 'wb') as local_file:
                ftp.retrbinary('RETR ' + remote_path, local_file.write)

            self.sql_insert_queue.put([file_name, file_uuidname, local_path, remote_path, file_format, file_size])
            # self.save_file_info(file_name, file_uuidname, local_path, remote_path, file_format, file_size)
            self.downloaded_file.add(remote_path)
            return True
        except Exception as e:
            if 'Broken pipe' in str(e):
                self.can_connect = False
                logging.error(f'{remote_path}：连接已断开，请检查网络')
                return False
            logging.error(f'{remote_path}：' + str(e))
        return False

    def is_any_thread_running(self):
        # logging.error(self.thread_running_flag.values())
        for status in self.thread_running_flag.values():
            if status:
                return True
        return False

    def scan_files(self):
        ftp = self.create_client()
        if not ftp:
            return
        thread_name = str(uuid.uuid4())
        self.thread_running_flag[thread_name] = False
        while (self.is_any_thread_running() or not self.dir_queue.empty()) and self.can_connect:
            try:
                try:
                    dir_path = self.dir_queue.get(timeout=0.1)
                except:
                    continue
                self.thread_running_flag[thread_name] = True
                self.get_files(dir_path, ftp)
            except Exception as e:
                if 'Broken pipe' in str(e):
                    self.can_connect = False
                    logging.error(f'连接已断开，请检查网络')
                    return False
                logging.error(f'{dir_path}：' + str(e))
            self.thread_running_flag[thread_name] = False
            if not self.scaned_path_nums % 500:
                logging.warning('-'*80)
                logging.warning(f'待扫描文件夹数量：{self.dir_queue.qsize()}')
                logging.warning(f'已扫描文件夹数量：{self.scaned_path_nums}')
                logging.warning(f'已下载文件数量：{len(self.downloaded_file)}')
                logging.warning(f'进程运行情况：{self.thread_running_flag.values()}')
                logging.warning('-'*80)
        ftp.close()
        self.thread_running_flag[thread_name] = False
    
    def download_file_thread(self):
        ftp = self.create_client()
        if not ftp:
            return
        thread_name = str(uuid.uuid4())
        self.thread_running_flag[thread_name] = False
        while (self.is_any_thread_running() or not self.dir_queue.empty() or not self.download_file_queue.empty()) and self.can_connect:
            try:
                try:
                    data = self.download_file_queue.get(timeout=0.1)
                except:
                    continue
                self.thread_running_flag[thread_name] = True
                self.download_file(data[0], data[1], ftp)
            except Exception as e:
                logging.error(f'Download【{data[0]}】Error, Filesize【{data[1]}】：' + str(e))
            self.thread_running_flag[thread_name] = False
        ftp.close()
        self.thread_running_flag[thread_name] = False
    
    def sql_insert_thread(self):
        thread_name = str(uuid.uuid4())
        self.thread_running_flag[thread_name] = False
        while (self.is_any_thread_running() or not self.dir_queue.empty() \
            or not self.download_file_queue.empty() or not self.sql_insert_queue.empty()) and self.can_connect:
            data_list = []
            try:
                for _ in range(self.sql_insert_num):
                    try:
                        data = self.sql_insert_queue.get(timeout=0.1)
                        data_list.append(data)
                    except:
                        continue
                if not data_list:
                    continue
                self.thread_running_flag[thread_name] = True
                self.save_file_info_many(data_list)
            except Exception as e:
                logging.error(f'sql_insert_thread：' + str(e))
            self.thread_running_flag[thread_name] = False
        self.thread_running_flag[thread_name] = False

def main():
    css = FTPClient(task_id=1, ip_id=1)
    threads = []
    threads.append(threading.Thread(target=css.download_file_thread))
    threads.append(threading.Thread(target=css.download_file_thread))
    threads.append(threading.Thread(target=css.sql_insert_thread))
    for _ in range(css.thread_nums):
        threads.append(threading.Thread(target=css.scan_files))

    
    start_time = time.time()
    for t in threads:
        t.start()
        time.sleep(0.5)

    for t in threads:
        t.join()
    end_time = time.time()
    logging.warning(f'耗时：{end_time - start_time}秒') 
    logging.warning('-'*80)
    logging.warning(f'待扫描文件夹数量：{css.dir_queue.qsize()}')
    logging.warning(f'已扫描文件夹数量：{css.scaned_path_nums}')
    logging.warning(f'已下载文件数量：{len(css.downloaded_file)}')
    logging.warning(f'进程运行情况：{css.thread_running_flag.values()}')
    logging.warning(f'连接情况：{css.can_connect}')
    logging.warning('-'*80)

if __name__ == '__main__':
    main()
    # css = FTPClient(task_id=1, ip_id=1)
    
    # a = css.list_dir_info(css.create_client(), '/home/tutu')
    # print(a)
    
    


